var Scraper = function(params){
	var sitemap = new Sitemap(params.db);
	var pageId = params.pageId;
	var data = {page: pageId, content: []};
	
	console.log(params);
	console.log(sitemap);
	var sourcePage = sitemap.getPage(pageId);
	
	this.scrape = function(){
		//collect fields
		var fields = sourcePage.fields;
		var links = sourcePage.linkedPages;
		
		if(sourcePage.rowElement && sourcePage.rowElement.selector){
			var rowElements = sourcePage.rowElement.select();
			for (var i = 0; i < rowElements.length; i++) {
				var rowElement = $(rowElements[i]);
				data.content.push(this.collectFieldAndLink(fields, links, rowElement));
			}
		}else{// one item
			data.content.push(this.collectFieldAndLink(fields, links));
		}
		
		//next page
		var paging = sourcePage.paging;
		if(paging && paging.type){
			if(paging.type == 'url'){
				data.nextPage = paging.nextPage();
			}else{
				data.nextPage = 'client';
			}
		}
		return data;
	}
	
	this.collectFieldAndLink = function(fields, links, rowElement){
		var o = {};
		for (var i = 0; i < fields.length; i++) {
			var pf = fields[i];
			o[pf.title] = pf.getValue(rowElement);
		}
		for (var i = 0; i < links.length; i++) {
			var lk = links[i];
			if(lk.paging.type){
				var content;
				if(lk.paging.type == 'url'){
					var content = lk.paging.nextPage(rowElement);
					//use rowelement
					if(!content){
						content = rowElement[0].href;
					}
				}else{
					content = 'client';
				}
			}
			
			o[lk.sourcePage.name] = {
				page: lk.sourcePage.id,
				content: content
			}
		}
		return o;
	}
}